#! /usr/bin/python2

# kjssar.py
# KJ's simplified stream archiver (kjssar)

# (C)Copyright 2013-2015 by Klaus-J. Wolf, kj (at) seismic (dot) de
# This file is licensed under the General Public License Version 2.
# http://www.gnu.org/licenses/gpl-2.0.html

# This is a proof-of-concept implementation.

###

# Comment: While this tool was started with only Python 2 available on the
# developer's platform, for some reason, it appears, that Python 2 is the
# only version of Python this specific task can be implemented in.


import sys, os, os.path, subprocess, stat, pwd, grp, hashlib, threading, re

import types

import xattr, posix1e


#############################################################################


SUPERID="\\KJJSAR:"
VERSION="00.0001"

HASHNAMES=["md5","sha512","ripemd160","whirlpool"]
#EXTRAS=["mtime","size"]

ERROUTF=sys.stderr
OUTFENCODING="utf-8"
SYSTEMENCODING="utf-8"

HASHBUFSIZE = 0x1000000
COMPRBUFSIZE = 0x10000


#############################################################################


# remark: DO NOT USE locale.*

def getlocalcoding(fn=None):
    e = os.getenv("LC_ALL", None)
    if not e:
        e = os.getenv("LC_CTYPE", None)
        if not e:
            e = os.getenv("LANG", None)    # this is pretty much unixoid
            if not e:
                e = "en_US.latin1"   # fallback
    i = e.find(".")
    if i>=0:
        e = e[i+1:]
    if e=="C" or e=="POSIX":
        e = "ascii"
    return e


#############################################################################


def dic2str(d):
    res = ""
    for k,v in d.items():
        res += str(k)+":="+str(v)+"\\;"
    return res


def str2dic(s):
    res = {}
    for t in s.split("\\;"):
        u = t.split(":=",1)
        res[u[0]] = u[1]
    return res


#############################################################################


outputformats = { '1': ("list of filenames", "$FN()\n") }


def namedformat(fmt, dic, cvtuppercase=True):
    coding=dic.get("CD", "ascii")
    re_var = re.compile("\\$([a-zA-Z0-9_:]+)\\(([^\\)]+)\\)")
    while True:
        r = re_var.search(fmt)
        if not r:
            break
        xk = r.group(1)
        if cvtuppercase:
            xk = xk.upper()
        if xk in dic:
            xs = dic[xk]
            xs = xs.decode(coding).encode(SYSTEMENCODING)
            try:
                xf = int(r.group(2))
            except ValueError:
                xf = 0
            if xf<0:
                # left padding
                xf = -xf
                xf = len(xs)-xf
                if xf>0:
                    xs = (xf*" ")+xs
            else:
                # right padding
                xf = len(xs)-xf
                if xf>0:
                        xs = xs+(xf*" ")
        else:
            xs = ""
        fmt = fmt[:r.start()] + xs + fmt[r.end()+1:]
    return fmt


#############################################################################


def readxattrvalues(fn, domain=""):
    res = {}
    attrs = xattr.listxattr(fn)
    for attr in attrs:
        if attr.startswith(domain):
            res[attr] = xattr.getxattr(fn, attr)
    return res

def escctl(s):
    return s.replace("\\","\\\\").replace("\n","\\x0a")

def deescctl(s):
    return s.replace("\\x0a","\n").replace("\\\\","\\")

def uidstr(n):
    e = pwd.getpwuid(n)
    if not e:
        return str(n)
    return str(n)+'='+e.pw_name

def gidstr(n):
    e = grp.getgrgid(n)
    if not e:
        return str(n)
    return str(n)+'='+e.gr_name

def splitidstr(s):
    s = s.split("=",1)
    if len(s)<2:
        return (int(s),None)
    else:
        return (int(s[0]),s[1])

def splituidstr(s, gonumeric=False):
    if not s:
        uid = os.geteuid()
    else:
        uid,uname = splitidstr(s)
        if uname and not gonumeric:
            e = pwd.getpwnam(uname)
            if e:
                return e.pw_uid
    return uid

def splitgidstr(s, gonumeric=False):
    if not s:
        gid = os.getegid()
    else:
        gid,gname = splitidstr(s)
        if gname and not gonumeric:
            e = grp.getgrnam(gname)
            if e:
                return e.gr_gid
    return gid

def deabs(pth):
    #if os.path.isabs(fn):
    if pth[0]=='/':      # os.sep = '/'
        pth = pth[1:]    # this is simple, but only works with Unix
    pth = os.path.normpath(pth)
    return pth


def errorreport(fn, e):
    ERROUTF.write(fn.encode(OUTFENCODING))
    ERROUTF.write(": ")
    ERROUTF.write(str(e))
    ERROUTF.write('\n')


#############################################################################


class HashesClass:
    
    def __init__(self, buf, hashes):
        self.buf = buf
        self.hashes = hashes
    
    def start(self):
        threads=[]
        for h in self.hashes:
            thr=threading.Thread(target=self.f, args=(h,))
            thr.start()
            threads.append(thr)
        for thr in threads:
            thr.join()
        for thr in threads:
            del thr
    
    def f(self, h):
        h.update(self.buf)


def gethashes(fn, fndest=None, blksz=HASHBUFSIZE):
    
    hashv=[]
    for hn in HASHNAMES:
        hashv.append(hashlib.new(hn))
    
    n=0
    if type(fn) is types.FileType:
        f=fn
        closef=False
    else:
        f=open(fn, "rb")
        closef=True
    if fndest:
        if type(fndest) is types.FileType:
            fdest=fndest
            closefdest=False
        else:
            fdest=open(fndest, "wb")   # Version 1 was: "xb"
            closefdest=True
    else:
        fdest=None
    while True:
        b=f.read(blksz)
        if not b:
            break
        x = HashesClass(b, hashv)
        x.start()
        #for h in hashv:
        #    h.update(b)
        n+=len(b)
        if fdest:
            fdest.write(b)
    if closef:
        f.close()
    if fdest and closefdest:
        fdest.close()
    
    res = {}
    for i in range(len(HASHNAMES)):
        hn = HASHNAMES[i]
        vh = hashv[i].hexdigest()
        res[hn] = vh
    return res


#############################################################################


smode_info = """\t\t'u' set uid by number / 'U' set uid by name
\t\t'g' set gid by number / 'G' set gid by name
\t\t's' setuid/setgid/sticky ok
\t\t'a' absolute symlinks / 'r' relative symlinks
\t\t'c','b' character, block devices
\t\t'S','F' sockets, fifos
\t\t'A' absolute files

\t\tdefault for root: 'UGsacbSF'
\t\tdefault for user: 'rSF'
"""

class FileMetaData:
    
    def __init__(self, fn=None):
        if fn:
            self.collect(fn)


    def setty(self):
        if stat.S_ISDIR(self.st_mode):
            self.ty = 'd'
        elif stat.S_ISCHR(self.st_mode):
            self.ty = 'c'
        elif stat.S_ISBLK(self.st_mode):
            self.ty = 'b'
        elif stat.S_ISREG(self.st_mode):
            self.ty = 'r'
        elif stat.S_ISFIFO(self.st_mode):
            self.ty = 'f'
        elif stat.S_ISLNK(self.st_mode):
            self.ty = 'n'
        elif stat.S_ISSOCK(self.st_mode):
            self.ty = 's'


    def collect(self, fn):
        self.fn = fn
        #fn = fn.encode(SYSTEMENCODING)
        st = os.lstat(fn)
        self.encoding = SYSTEMENCODING
        self.acl = posix1e.ACL(file=fn).to_any_text(separator=',')
        self.xattr = dic2str(readxattrvalues(fn))
        self.st_mode = st.st_mode
        self.st_uid = st.st_uid
        self.st_gid = st.st_gid
        self.st_size = st.st_size
        self.st_mtime = st.st_mtime
        self.st_ctime = st.st_ctime
        self.st_rdev_min = os.minor(st.st_rdev)
        self.st_rdev_maj = os.major(st.st_rdev)
        self.setty()
        if self.ty=='n':
            self.lnk = os.readlink(fn)
    
    # function returns file handle if ty=='r'
    # 
    
    def stamp(self, fn, smode="", umode=""):
        if 'A' not in smode:
            fn = deabs(fn)
        try:
            st = os.lstat(fn)
        except OError as e:
            if e.errno==2:   # file does not exist
                st = None
            else:
                # probably permission denied (errno==13)
                errorreport(fn,e)
                return None
        if st:
            if umode=='k':
                return None
            if umode=='u':
                if self.st_mtime <= st.st_mtime:
                    return None
        if self.ty=='r':
            try:
                f = open(fn, "wb")
            except IOError as e:
                # probably permission denied (errno==13)
                errorreport(fn,e)
                return None
            return f
        elif self.ty=='d':
            try:
                os.mkdir(fn, st.st_mode & 0o777)
            except OSError as e:
                errorreport(fn,e)
                return None
            return True
        elif self.ty=='c' or self.ty=='b' or self.ty=='f':
            try:
                os.mknod(fn, st.st_mode, os.makedev(self.st_rdev_maj, self.st_rdev_min))
            except OSError as e:
                errorreport(fn,e)
                return None
        elif self.ty=='n':
            try:
                os.symlink(self.lnk, fn)
            except OSError as e:
                errorreport(fn,e)
                return None
        elif self.ty=='s':
            pass
        
        if self.ty!='n':
            os.chown(fn, self.st_uid, self.st_gid)
            os.chmod(fn, self.st_mode & 0o7777)
            posix1e.ACL(text=self.acl).applyto(fn)
            for k,v in str2dic(self.xattr).items():
                xattr.setxattr(fn, k, v)


    def write(self, ostr):
        ostr.write("\\EBEG\n")
        ostr.write("CD:")
        ostr.write(SYSTEMENCODING)
        ostr.write('\n')
        ostr.write("FN:")
        ostr.write(escctl(self.fn))   # use local encoding here, unchanged
        ostr.write('\n')
        ostr.write("MODE:")
        ostr.write(oct(self.st_mode))
        ostr.write('\n')
        ostr.write("UID:")
        ostr.write(uidstr(self.st_uid))
        ostr.write('\n')
        ostr.write("GID:")
        ostr.write(gidstr(self.st_gid))
        ostr.write('\n')
        ostr.write("SIZE:")
        ostr.write(str(self.st_size))
        ostr.write('\n')
        ostr.write("MTIME:")
        ostr.write(str(self.st_mtime))
        ostr.write('\n')
        ostr.write("CTIME:")
        ostr.write(str(self.st_ctime))
        ostr.write('\n')
        
        ostr.write("XATTR:")
        ostr.write(self.xattr)
        ostr.write('\n')
        ostr.write("ACL:")
        ostr.write(self.acl)
        ostr.write('\n')
        
        if self.ty=='b' or self.ty=='c':
            #if self.ty=='b':
            #    ostr.write("BDEV:")
            #else:
            #    ostr.write("CDEV:")
            ostr.write("DEV:")
            ostr.write(str(self.st_rdev_maj))
            ostr.write(',')
            ostr.write(str(self.st_rdev_min))
            ostr.write('\n')
        elif self.ty=='n':
            ostr.write("SYMLINK:")
            ostr.write(escctl(self.lnk))   # use local encoding here, unchanged
            ostr.write('\n')
        ostr.write("\\EEND\n")


    def hascontent(self):
        return self.ty=='r'


    def read(self, istr, smode=""):
        buf = {}
        while True:
            s = istr.readline()
            if not s:
                break
            if len(s)>0 and s[-1]=='\n':
                s = s[:-1]
            if not s:
                continue
            if s=="\\EBEG":
                continue
            if s=="\\EEND":
                break
            s=s.split(':',1)
            if len(s)!=2:
                continue
            buf[s[0]]=s[1]
        
        if not buf:
            return False
        
        self.encoding = buf.get("CD",None)
        self.fn = buf.get("FN",None)
        if self.fn:
            self.fn = deescctl(self.fn) # local encoding of the packing host

        self.lnk = deescctl(buf.get("SYMLINK", None)) # local encoding
        s = buf.get("DEV",None)
        if s:
            s = s.split(",")
            if len(s)==2:
                self.st_rdev_maj = int(s[0])
                self.st_rdev_min = int(s[1])
            
        self.st_mode = int(buf.get("MODE", "0700"), 8)   # should use os.umask() here, but that one's broken
        if "u" in smode or "U" in smode:
            self.st_uid = splituidstr(buf.get("UID", ""), "u" in smode)
        else:
            self.st_uid = os.geteuid()
        if "g" in smode or "G" in smode:
            self.st_gid = splitgidstr(buf.get("GID", ""), "g" in smode)
        else:
            self.st_gid = os.getegid()
        self.st_size = int(buf.get("SIZE", "0"))
        self.st_mtime = int(buf.get("MTIME", "0"))
        self.st_ctime = int(buf.get("CTIME", "0"))
        
        self.setty()
        
        self.rawbuf = buf
        
        return True


#############################################################################


def datadump(fn, outf):
    hashes = gethashes(fn, outf)
    outf.write("\\HBEG\n")
    for hn,hv in hashes.items():
        outf.write("HASH.")
        outf.write(hn)
        outf.write(':')
        outf.write(hv)
        outf.write('\n')
    outf.write("\\HEND\n")


def readhashblock(f):
    buf = {}
    while True:
        s = istr.readline()
        if not s:
            break
        if len(s)>0 and s[-1]=='\n':
            s = s[:-1]
        if not s:
            continue
        if s=="\\HBEG":
            continue
        if s=="\\HEND":
            break
        s=s.split(':',1)
        if len(s)!=2:
            continue
        buf[s[0]]=s[1]
    return buf


#############################################################################


def seekorread(f, sz, docksum, doseek):
    assert not (docksum and doseek), "cannot seek on pipe streams"
    if doseek:
        f.seek(sz, 1)
        return None
    if not docksum:
        n = sz
        while n>0:
            if n>0x4000:
                b = f.read(0x4000)
            else:
                b = f.read(n)
            if not b:
                break
            n -= len(b)
        return None
    else:
        return datadump(f)


#############################################################################


def byext(fn1, fn2):
    i1 = fn1.rfind('.')
    i2 = fn2.rfind('.')
    if i1>=0:
        j1 = fn1.rfind('/',i1+1)
        if j1>=0:
            i1=-1
    if i2>=0:
        j2 = fn2.rfind('/',i2+1)
        if j2>=0:
            i2=-1
    if i1>=0:
        fn1 = fn1[i1:]+"::"+fn1[:i1]
    if i2>=0:
        fn2 = fn2[i2:]+"::"+fn2[:i2]
    return cmp(fn1,fn2)


#############################################################################


def __readerThread(src, dest, bufsz):
    while True:
        b=src.read(bufsz)
        if not b:
            break
        dest.write(b)


iotask = None
realfile = None


def myopen(fn, fmode, xfilter=None):
    global iotask, realfile
    if not xfilter:
        iotask = None
        realfile = None
        return open(fn, fmode)
    realfile = open(fn, fmode)
    if fmode[0]=='r':
        p = subprocess.Popen(xfilter+" -d", bufsize=COMPRBUFSIZE, stdin=subprocess.PIPE, stdout=subprocess.PIPE, close_fds=False)
        iotask = threading.Thread(target=__readerThread, args=(realfile,p.stdin,COMPRBUFISZE))
        iotask.start()
        return p.stdout
    elif fmode[0]=='w':
        p = subprocess.Popen(xfilter, bufsize=COMPRBUFSIZE, stdin=subprocess.PIPE, stdout=subprocess.PIPE, close_fds=False)
        iotask = threading.Thread(target=__readerThread, args=(p.stdout,realfile,COMPRBUFISZE))
        iotask.start()
        return p.stdin


def myclose(f):
    if not iotask:
        f.close()
    else:
        iotask.join()
        iotask.stdin.close()
        iotask.stdout.close()
        realfile.close()


#############################################################################


# cmode : creation / display / extraction / append
# umode : overwrite / update
# smode : setuid,gid, ...
# csortmode : 'e'=by extension


class Archive:
    
    def __init__(self, fn, cmode, umode="", smode="", csortmode="", xproc=None, tformat="1"):
        self.fn = fn
        self.cmode = cmode
        self.umode = umode
        self.smode = smode
        self.csortmode = csortmode
        self.xproc = xproc
        self.tformat = tformat
        self.objs = []


    def add(self, root):
        if not os.path.isdir(root):
            self.objs.append(root)
        else:
            for r, ds, fs in os.walk(root):
                for fn in ds:
                    ffn = os.path.join(r,fn)
                    #ffn = ffn.decode(SYSTEMENCODING)
                    self.objs.append(ffn)
                for fn in fs:
                    ffn = os.path.join(r,fn)
                    #ffn = ffn.decode(SYSTEMENCODING)
                    self.objs.append(ffn)


    def execute(self, unicodemode=True):
        if cmode=='c' or cmode=='a':
            if cmode=='a' and os.path.exists(fn):
                if self.xproc:
                    print >>sys.stderr, "cannot use append on compressed files"
                    sys.exit(9)
                f = open(self.fn, "ab")
            else:
                f = myopen(self.fn, "wb", self.xproc)
                f.write(SUPERID)
                f.write(VERSION)
                f.write('\n')
            if 'e' in self.csortmode:
                self.objs.sort(byext)
            for o in self.objs:
                fm = FileMetaData(o)
                fm.write(f)
                if fm.hascontent():
                    datadump(o, f)
            myclose(f)
        elif cmode=='t':
            f = myopen(self.fn, "rb", self.xproc)
            s = f.readline()
            if not s.startswith(SUPERID):
                print >>sys.stderr, "unknown file format"
                sys.exit(8)
            while True:
                fm = FileMetaData()
                if not fm.read(f):
                    break
                s = namedformat(outputformats[self.tformat], fm.rawbuf)
                sys.stdout.write(s)
                sys.stdout.write('\n')
                if fm.hascontent():
                    seekorread(f, fm.st_size, docksum=False, doseek=not self.xproc) # no result
                    readhashblock(f) # throw result away
            f.close()
        elif cmode=='x':
            f = myopen(self.fn, "rb", self.xproc)
            s = f.readline()
            if not s.startswith(SUPERID):
                print >>sys.stderr, "unknown file format"
                sys.exit(8)
            while True:
                fm = FileMetaData()
                if not fm.read(f):
                    break
                #s = namedformat(outputformats[self.tformat], fm.rawbuf)
                #sys.stdout.write(s)
                #sys.stdout.write('\n')
                if fm.hascontent():
                    nf = open( # TODO
                    filecopy(f, 
                    seekorread(f, fm.st_size, docksum=False, doseek=not self.xproc) # no result
                    readhashblock(f) # throw result away
                fm.stamp(fn)
            f.close()


#############################################################################


if __name__=="__main__":

    if len(sys.argv)<4:
    
        print "KJ's simplified stream archiver (kjssar), Version", VERSION
        print
        print "(C)Copyright 2013-2015 by Klaus-J. Wolf, kj (at) seismic (dot) de"
        print "This program is licensed under the General Public License Version 2."
        print
        print "usage:", sys.argv[0], "-c|x|t|a", "[options]", "archive", "root..."
        print "\t-k\tkeep old files (eXtraction)"
        print "\t-u\tupdate files (eXtraction)"
        print "\t-U\tuse Unicode file names (eXtraction)"
        print "\t-S\tsort by extension (Creation, Appending)"
        print "\t-X <compressor>"
        print "\t-T<format>\t(display=T)"
        for k,v in outputformats.items():
            print "\t\t%-20s %s" % (k,v[0])
        print "\t-O<smode>\t(eXtraction)"
        print smode_info
        sys.exit(1)

    SYSTEMENCODING = getlocalcoding("/")
    compressor=None
    umode='o'
    cmode=None
    csortmode=""
    tformat="1"
    unicodemode = SYSTEMENCODING.lower() in ["utf_8","utf-8","u8","utf","utf8"] # see Python standard encodings
    
    if os.geteuid()==0:
        smode = "UGsacbSF"
    else:
        smode = "rSF"
    
    n=1
    
    if sys.argv[n]=='-c':
        cmode='c'
    elif sys.argv[n]=='-x':
        cmode='x'
    elif sys.argv[n]=='-t':
        cmode='t'
    elif sys.argv[n]=='-a':
        cmode='a'
    else:
        print >>sys.stderr, "must give primary command"
        sys.exit(9)

    n+=1
    
    while sys.argv[n].startswith('-'):
        if len(sys.argv[n])<2:
            print >>sys.stderr, "option missing"
            sys.exit(9)
        if sys.argv[n][1]=='k':
            umode='k'
        elif sys.argv[n][1]=='u':
            umode='u'
        elif sys.argv[n][1]=='S':
            csortmode='e'
        elif sys.argv[n][1]=='X':
            if len(sys.argv[n])>2:
                compressor=sys.argv[n][2:]
            else:
                n+=1
                if n==len(sys.argv):
                    print >>sys.stderr, "too few arguments"
                    sys.exit(9)
                compressor = sys.argv[n]
        elif sys.argv[n][1]=='O':
            smode = sys.argv[n][2:]
        elif sys.argv[n][1]=='U':
            unicodemode=True
        elif sys.argv[n][1]=='T':
            tformat = sys.argv[n][2:]
            if tformat not in outputformats:
                print >>sys.stderr, "unknown format:", sys.argv[n][2:]
                sys.exit(2)
        else:
            print >>sys.stderr, "unknown option:", sys.argv[n]
            sys.exit(2)
        n+=1

    arc = Archive(sys.argv[n], cmode, umode, smode, csortmode, compressor, tformat)
    for i in range(n+1, len(sys.argv)):
        arc.add(sys.argv[i])
    arc.execute(unicodemode)


# TODO return error level on I/O errors
# TODO if ACLs are present, attach U/G list
# TODO primary commands as normal options
# TODO sparse files, hardlinks
# TODO -F files from...
# TODO typical : UID/GID by name, if not resolvable, $uid/$gid (user) or UID/GID by number (root)
# TODO extraction in mtree mode
# TODO no-overwrite
